package com.it.bigdata;

import com.it.bigdata.domain.TieZi;
import com.it.bigdata.download.impl.BaiDuTieBaDownLoad;
import com.it.bigdata.parse.impl.BaiDuTieBaParser;
import com.it.bigdata.schedual.impl.BaiDuTieBaSchedualForQueue;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class Splider {
    public static void main(String[] args) {
        /**
         * 下载页面
         *      HttpClient用来创建下载
         * 解析页面
         *      htmlCleaner用来解析下载下来的页面(讲不规则的html页面变成规则的xml文件)
         *          解析出来的文件放到队列中
         *              queue队列
         *              Redis中的list队列
         * 存储数据
         */
        BaiDuTieBaDownLoad baiDuTieBaDownLoad = new BaiDuTieBaDownLoad();
        BaiDuTieBaParser baiDuTieBaParser = new BaiDuTieBaParser();
        BaiDuTieBaSchedualForQueue bdSchedual = BaiDuTieBaSchedualForQueue.getBdQueue();
        ExecutorService executorService = Executors.newFixedThreadPool(2);
        executorService.execute(new Runnable() {
            @Override
            public void run() {
                //下载页面
                String html = baiDuTieBaDownLoad.download("https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn=" + bdSchedual.getCurrentPageNum());
                bdSchedual.nextPage();
                //解析页面
                baiDuTieBaParser.parsePageList(html);
                //System.out.println(bdSchedual.pop().toString());
            }
        });
        /**
         * 下载并解析具体的页面
         */
        executorService.execute(new Runnable() {

            @Override
            public void run() {
                while (true){
                    String pageUrl = bdSchedual.pop();
                    //下载具体的页面
                    if (pageUrl != null) {
                        String html = baiDuTieBaDownLoad.download("https://tieba.baidu.com" + pageUrl);
                       // System.out.println(html);
                        //解析具体页面
                        TieZi tieZi = baiDuTieBaParser.parseCurrentToTiezi(html);
                        System.out.println(tieZi);
                    }
                }

            }
        });
    }
}
